{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.1 常见处理HTML文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Element html at 0x22ef5060f48>\n"
     ]
    }
   ],
   "source": [
    "from lxml import etree\n",
    "\n",
    "str='''\n",
    "<html>\n",
    "    <body>\n",
    "        <a>link</a>\n",
    "        <div class='container' id='first'>\n",
    "            <p class='common' id='two'>hello world</p>\n",
    "            <p class='common' id='three'>I love python</p>\n",
    "        </div>\n",
    "    </body>\n",
    "</html>\n",
    "'''\n",
    "\n",
    "html=etree.HTML(str)#自动修正为标准html\n",
    "print(html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element div at 0x22ef5157f88>]\n"
     ]
    }
   ],
   "source": [
    "res1=html.xpath(\"//*[@id='first']\")\n",
    "print(res1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element p at 0x22ef5176048>]\n"
     ]
    }
   ],
   "source": [
    "res2=html.xpath(\"/html/body/div/p[@id='two']\")\n",
    "print(res2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element p at 0x22ef5176048>]\n"
     ]
    }
   ],
   "source": [
    "res3=html.xpath(\"//p[@id='two' and @class='common']\")\n",
    "print(res3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I love python']\n"
     ]
    }
   ],
   "source": [
    "res3=html.xpath(\"/html/body/div/p[@id='three']/text()\")\n",
    "print(res3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I love python']\n"
     ]
    }
   ],
   "source": [
    "res4=html.xpath(\"//p[@id='three']/text()\")\n",
    "print(res4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1.2 常见处理XML文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Element r at 0x23e6ed555c8>\n"
     ]
    }
   ],
   "source": [
    "from lxml import etree\n",
    "\n",
    "str =\"\"\"\n",
    "<r>\n",
    "  <e a=\"1\"/>\n",
    "  <f a=\"2\" b=\"1\">hello python</f>\n",
    "  <f/>\n",
    "  <g>\n",
    "    <i c=\"2\">hello xpath</i>\n",
    "    世界你好\n",
    "    <j>I love you</j>\n",
    "  </g>\n",
    "</r>\n",
    "\"\"\"\n",
    "\n",
    "xml=etree.XML(str)\n",
    "\n",
    "print(xml)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element e at 0x23e6eee8188>]\n"
     ]
    }
   ],
   "source": [
    "res5=xml.xpath('/r/e')\n",
    "print(res5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['hello python']\n"
     ]
    }
   ],
   "source": [
    "res6=xml.xpath('/r/f/text()')\n",
    "print(res6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello python\n"
     ]
    }
   ],
   "source": [
    "res7=xml.xpath('string(/r/f)')\n",
    "print(res7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 查找具有特定属性的节点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Element place at 0x23e6ed51708>\n"
     ]
    }
   ],
   "source": [
    "from lxml import etree\n",
    "\n",
    "str='''\n",
    "<place>\n",
    "    <name>zhangsan</name>\n",
    "    <plan name=\"北京\" type=\"first\"/>\n",
    "    <plan name=\"上海\" type=\"second\"/>\n",
    "</place>\n",
    "'''\n",
    "\n",
    "xml=etree.XML(str)\n",
    "\n",
    "print(xml)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element plan at 0x23e6eeebb88>, <Element plan at 0x23e6eeebbc8>]\n"
     ]
    }
   ],
   "source": [
    "#匹配所有name属性的标签\n",
    "res1=xml.xpath('/place/*[@name]')\n",
    "print(res1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element plan at 0x23e6eeebb88>, <Element plan at 0x23e6eeebbc8>]\n"
     ]
    }
   ],
   "source": [
    "res2=xml.xpath('//*[@name]')\n",
    "print(res2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 通过属性值的字段来查找节点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element plan at 0x23e6eeebb88>]\n"
     ]
    }
   ],
   "source": [
    "res3=xml.xpath(r\"/place/*[contains(@name,'北')]\")\n",
    "print(res3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element plan at 0x23e6eeebb88>]\n"
     ]
    }
   ],
   "source": [
    "res4=xml.xpath(\"//*[contains(@name,'北')]\")\n",
    "print(res4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 属性查询"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['北京', '上海']\n"
     ]
    }
   ],
   "source": [
    "res5=xml.xpath('//place/plan/@name')\n",
    "print(res5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 处理HTML常用方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<html><body><div>\n",
      "    <ul>\n",
      "         <li class=\"first\"><a href=\"https://www.csdn.net/\">CSDN</a></li>\n",
      "         <li class=\"two\"><a href=\"https://www.zhihu.com/hot\">zhihu</a></li>\n",
      "         <li class=\"three\"><a href=\"https://www.runoob.com/linux/linux-tutorial.html\" class=\"linux\">linux</a></li>\n",
      "         <li class=\"four\"><a href=\"https://leetcode-cn.com/\">leecode</a></li>\n",
      "         <li class=\"five\"><a href=\"https://www.facebook.com/\">facebook</a></li>\n",
      "         <li class=\"six\"><a href=\"https://www.bilibili.com/\">bilibili</a></li>\n",
      "     </ul>\n",
      " </div></body></html>\n"
     ]
    }
   ],
   "source": [
    "from lxml import etree\n",
    "\n",
    "html = '''\n",
    "<div>\n",
    "    <ul>\n",
    "         <li class=\"first\"><a href=\"https://www.csdn.net/\">CSDN</a></li>\n",
    "         <li class=\"two\"><a href=\"https://www.zhihu.com/hot\">zhihu</a></li>\n",
    "         <li class=\"three\"><a href=\"https://www.runoob.com/linux/linux-tutorial.html\" class=\"linux\">linux</a></li>\n",
    "         <li class=\"four\"><a href=\"https://leetcode-cn.com/\">leecode</a></li>\n",
    "         <li class=\"five\"><a href=\"https://www.facebook.com/\">facebook</a></li>\n",
    "         <li class=\"six\"><a href=\"https://www.bilibili.com/\">bilibili</a></li>\n",
    "     </ul>\n",
    " </div>'''\n",
    "\n",
    "#数据转换成标签树的方式一\n",
    "html_tree = etree.HTML(html)\n",
    "# print(html_tree)\n",
    "#看一下html_tree里面的数据\n",
    "print(etree.tostring(html_tree).decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element li at 0x22ef5189dc8>, <Element li at 0x22ef5189448>, <Element li at 0x22ef5189b08>, <Element li at 0x22ef5189fc8>, <Element li at 0x22ef5189d48>, <Element li at 0x22ef5189688>]\n"
     ]
    }
   ],
   "source": [
    "#xpath返回数据是列表  [Element li 内存地址]\n",
    "li = html_tree.xpath('//li')\n",
    "print(li)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['zhihu']\n"
     ]
    }
   ],
   "source": [
    "#获取属性为two文本,li2[1]返回的是Element,继续使用xpath\n",
    "li2 = html_tree.xpath('//li[@class=\"two\"]')\n",
    "print(li2[0].xpath('.//a/text()'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element li at 0x22ef5189448>, <Element li at 0x22ef5189b08>, <Element li at 0x22ef5189fc8>, <Element li at 0x22ef5189d48>, <Element li at 0x22ef5189688>]\n"
     ]
    }
   ],
   "source": [
    "#查询class属性不等于'first'标签\n",
    "li3 = html_tree.xpath('//li[@class!=\"first\"]')\n",
    "print(li3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element li at 0x22ef5189448>]\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "#查询li标签class包含tw字符串\n",
    "li4 = html_tree.xpath('//li[contains(@class,\"tw\")]')\n",
    "print(li4)\n",
    "print(len(li4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['https://www.zhihu.com/hot']\n"
     ]
    }
   ],
   "source": [
    "#获取li4标签的属性href\n",
    "h=li4[0].xpath('./a/@href')\n",
    "print(h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['https://www.csdn.net/', 'https://www.zhihu.com/hot', 'https://www.runoob.com/linux/linux-tutorial.html', 'linux', 'https://leetcode-cn.com/', 'https://www.facebook.com/', 'https://www.bilibili.com/']\n"
     ]
    }
   ],
   "source": [
    "#获取所有a标签中所有的属性\n",
    "c=html_tree.xpath('//li/a/@*')\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element li at 0x22ef5189dc8>, <Element li at 0x22ef5189b08>, <Element li at 0x22ef5189fc8>, <Element li at 0x22ef5189d48>, <Element li at 0x22ef5189688>]\n",
      "<li class=\"first\"><a href=\"https://www.csdn.net/\">CSDN</a></li>\n",
      "         \n"
     ]
    }
   ],
   "source": [
    "#查询li标签,查询class不包含tw字符串的标签\n",
    "li5 = html_tree.xpath('//li[not(contains(@class,\"tw\"))]')\n",
    "print(li5)\n",
    "print(etree.tostring(li5[0]).decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element li at 0x22ef5189b08>]\n",
      "<li class=\"first\"><a href=\"https://www.csdn.net/\">CSDN</a></li>\n",
      "         \n"
     ]
    }
   ],
   "source": [
    "# two#查询li标签,class不包含f字符串,同时包含属性three\n",
    "li6 = html_tree.xpath('//li[not(contains(@class,\"f\"))][@class=\"three\"]')\n",
    "print(li6)\n",
    "print(etree.tostring(li[0]).decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element li at 0x22ef5189dc8>, <Element li at 0x22ef5189448>, <Element li at 0x22ef5189b08>, <Element li at 0x22ef5189fc8>, <Element li at 0x22ef5189d48>, <Element li at 0x22ef5189688>]\n",
      "[<Element li at 0x22ef5189dc8>, <Element li at 0x22ef5189448>, <Element li at 0x22ef5189b08>, <Element li at 0x22ef5189fc8>, <Element li at 0x22ef5189d48>, <Element li at 0x22ef5189688>]\n"
     ]
    }
   ],
   "source": [
    "#查找根目录下所有li\n",
    "ll = html_tree.xpath('/html/body/div/ul/li')\n",
    "ll2=html_tree.xpath('//li')\n",
    "print(ll)\n",
    "print(ll2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element li at 0x22ef5189688>]\n",
      "[<Element li at 0x22ef5189d48>]\n"
     ]
    }
   ],
   "source": [
    "#查询li标签最后一个\n",
    "li7 = html_tree.xpath('//li[last()]')\n",
    "li8=html_tree.xpath('//li[5]')\n",
    "print(li7)\n",
    "print(li8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<li class=\"five\"><a href=\"https://www.facebook.com/\">facebook</a></li>\n",
      "         \n"
     ]
    }
   ],
   "source": [
    "#查询li标签倒数第二个具体内容\n",
    "li9 = html_tree.xpath('//li[last()-1]')\n",
    "print(etree.tostring(li9[0]).decode('utf-8'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element li at 0x22ef5189dc8>, <Element li at 0x22ef5189448>]\n"
     ]
    }
   ],
   "source": [
    "#查询位置小于3的标签\n",
    "li10 = html_tree.xpath('//li[position()<3]')\n",
    "print(li10)\n",
    "# print(etree.tostring(li10[0]).decode('utf-8'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
